package BF;

import redis.clients.jedis.Jedis;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class Urls {

    public static List<String> urlList=new ArrayList<String>();
    public int count;

    public Jedis jedis=null;
    public static void main(String[] args) throws IOException {
        /*
         * 获取单网页里所有超链接里的href属性值并打印到控制台
         */
//      getUrl("");

    }

    public  Urls(){
        loadRedis();
    }

    public void loadRedis(){
        if (jedis==null){
            jedis = new Jedis("192.168.31.216");
            System.out.println("连接成功");
        }


        //设置 redis 字符串数据
        try {

            jedis.set("runoobkey", "www.runoob.com");
            jedis.set("url:Count","0");
            jedis.del("url:List");
            // 获取存储的数据并输出
            System.out.println("redis 存储的字符串为: "+ jedis.get("runoobkey"));
        }catch (Exception ex){
            System.out.println("redis 失败");
        }


    }

    public boolean isIn(String url){
        List<String> args=new ArrayList<String>();
        args.add("urls");
        args.add(url);
        Object o= jedis.eval("return redis.call('bf.add',KEYS[1],KEYS[2])",args,args);
//        Object o= jedis.eval("return redis.call('bf.add',KEYS[1],KEYS[2])",args,args);
//        System.out.println(Integer.valueOf(o.toString()));


        return Integer.valueOf(o.toString())==0;
    }

    public URL url;

    public  void NewgetUrl(String urlString){
        if (urlString.isEmpty()){
            urlString="https://www.91caiqing.com";
        }
        try {
            url = new URL(urlString);

            StringBuilder sb = new StringBuilder();

            HttpURLConnection htpcon = (HttpURLConnection) url.openConnection();
            htpcon.setRequestMethod("GET");
            htpcon.setRequestProperty("Content-Type", "text/html;charset=utf-8");
            htpcon.setRequestProperty("user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36");

            htpcon.setDoOutput(true);
            htpcon.setDoInput(true);
            htpcon.setUseCaches(false);
            htpcon.setConnectTimeout(10000);
            htpcon.setReadTimeout(10000);
            InputStream in = htpcon.getInputStream();
//            BufferedReader  br = new BufferedReader(new InputStreamReader(url.openStream(),Charset.forName("gbk")));
            BufferedReader  br = new BufferedReader(new InputStreamReader(in));
            String temp = "";
            //    BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream("E:/163.txt"),Charset.forName("gbk")));
            while((temp=br.readLine() )!= null){
                sb.append(temp);
                sb.append("\n");
            }

            //[\\s\\s] 匹配所有字符
            String regStr ="<a(?:[^<]*?href=\"(http[\\s\\S]+?)\"[\\s\\S]*?)</a>?";
            Pattern p = Pattern.compile(regStr);
            Matcher m = p.matcher(sb.toString());
            while(m.find()){

                String newUrl=m.group(1);

                if (!newUrl.isEmpty() && !isIn(newUrl)){
                    try {

                        jedis.hset("url:List", newUrl,"1");
                        jedis.incr("url:Count");
                    }catch (Exception ex){
                        System.out.println("Redis 操作失败!");
                    }

                    System.out.println(count++ +" ---- "+newUrl);

                    NewgetUrl(newUrl);

                }


            }

        }catch (Exception ex){

        }



    }


    public  void getUrl(String urlString,BloomFilter bloomFilter){
        if (urlString.isEmpty()){
            urlString="https://www.91caiqing.com";
        }
        try {
            url = new URL(urlString);

            StringBuilder sb = new StringBuilder();

            HttpURLConnection htpcon = (HttpURLConnection) url.openConnection();
            htpcon.setRequestMethod("GET");
            htpcon.setDoOutput(true);
             htpcon.setDoInput(true);
             htpcon.setUseCaches(false);
             htpcon.setConnectTimeout(10000);
             htpcon.setReadTimeout(10000);
            InputStream in = htpcon.getInputStream();
//            BufferedReader  br = new BufferedReader(new InputStreamReader(url.openStream(),Charset.forName("gbk")));
            BufferedReader  br = new BufferedReader(new InputStreamReader(in));
            String temp = "";
            //    BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream("E:/163.txt"),Charset.forName("gbk")));
            while((temp=br.readLine() )!= null){
                sb.append(temp);
                sb.append("\n");
            }

            //[\\s\\s] 匹配所有字符
            String regStr ="<a(?:[^<]*?href=\"(http[\\s\\S]+?)\"[\\s\\S]*?)</a>?";
            Pattern p = Pattern.compile(regStr);
            Matcher m = p.matcher(sb.toString());
            while(m.find()){
//            System.out.println(m.group().replaceAll("\\s+", " "));//打印出匹配子序列[把多空格或者换号或者多tab替换成一个空格，便于查看]

                String newUrl=m.group(1);

//                if (newUrl.indexOf("91caiqing.com")!=-1 && newUrl.indexOf("91caiqing.com/content/map")==-1){
//                    count++;
                    if (!newUrl.isEmpty() && !bloomFilter.contains(newUrl)){
                        try {

                            jedis.hset("url:List", newUrl,"1");
                            jedis.incr("url:Count");
                        }catch (Exception ex){
                            System.out.println("Redis 操作失败!");
                        }

                        System.out.println(count++ +" ---- "+newUrl);
                        bloomFilter.add(newUrl);
                        getUrl(newUrl,bloomFilter);

                    }

//                }


            }

        }catch (Exception ex){

        }



    }
}
